package common.main.cnki;


import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.utils.DomTree;
import common.utils.MD5Util;
import org.apache.log4j.Logger;
import org.w3c.dom.*;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.List;


public class CnikMain {

    //String url = "http://kns.cnki.net/kns/brief/Default_Result.aspx?code=CJFQ&kw=<keyword>&korder=0&sel=1";
    public static String urlPrefix = "http://kns.cnki.net";
    public static Logger logger = Logger.getLogger(CnikMain.class);

    public static SimpleHttp http = new SimpleHttp();
    public static HtmlInfo html = new HtmlInfo();

    public static int requestSameListUrl = 0;//当下一页url为空时，重新获取本页信息一次。

    public static void main(String[] args) throws InterruptedException, MalformedURLException {
        //
        CnkiConfig.init();

        String allMD5 = CnkiSaveDataToSql.getAllCnkiMd5();

        html.setEncode("utf-8");
        String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
        html.setUa(ua);

        //String url = "http://kns.cnki.net/kns/request/SearchHandler.ashx?action=&NaviCode=*&ua=1.11&PageName=ASP.brief_default_result_aspx&DbPrefix=CJFQ&DbCatalog=%e4%b8%ad%e5%9b%bd%e5%ad%a6%e6%9c%af%e6%96%87%e7%8c%ae%e7%bd%91%e7%bb%9c%e5%87%ba%e7%89%88%e6%80%bb%e5%ba%93&ConfigFile=SCDBINDEX.xml&db_opt=CJFQ&txt_1_sel=SU%24%25%3D%7C&txt_1_value1=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD&txt_1_special1=%25&his=0&parentdb=SCDB&__=Mon%20Aug%2006%202018%2011%3A18%3A52%20GMT%2B0800%20(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)";

        String firstAuthorUrl = "http://kns.cnki.net/kcms/detail/knetsearch.aspx?sfield=au&skey=<authorName>&code=<authorCode>";
        String journalUrl = "http://navi.cnki.net/KNavi/JournalDetail?pcode=<1>&pykm=<2>&Year=&Issue=&Entry=";
        String citeAuthorUrl = "http://kns.cnki.net/kcms/detail/frame/asynlist.aspx?curdbcode=CJFQ&reftype=601&catalogId=lcatalog_func601&catalogName=关联作者&";
        String yearsCiteNumUrl = "http://kns.cnki.net/kcms/detail/block/refyear.aspx?vl=&";
        //todo 采集完成后修改时间段
        String getYearInfo = "http://kns.cnki.net/kns/request/SearchHandler.ashx?action=&NaviCode=*&ua=1.21&PageName=ASP.brief_result_aspx&DbPrefix=CJFQ&DbCatalog=%e4%b8%ad%e5%9b%bd%e5%ad%a6%e6%9c%af%e6%9c%9f%e5%88%8a%e7%bd%91%e7%bb%9c%e5%87%ba%e7%89%88%e6%80%bb%e5%ba%93&ConfigFile=CJFQ.xml&db_opt=CJFQ&db_value=%E4%B8%AD%E5%9B%BD%E5%AD%A6%E6%9C%AF%E6%9C%9F%E5%88%8A%E7%BD%91%E7%BB%9C%E5%87%BA%E7%89%88%E6%80%BB%E5%BA%93&year_from=2005&year_to=2006&year_type=echar&txt_1_sel=SU&txt_1_value1=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD&txt_1_relation=%23CNKI_AND&txt_1_special1=%3D&his=0&__=Wed%20Aug%2008%202018%2014%3A28%3A22%20GMT%2B0800%20(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)";

        String url = "";
        boolean isFirst = true;
        boolean isRestart = false;
        if(CnkiConfig.entranceUrl != null){
            url = CnkiConfig.entranceUrl;
            isFirst = false;
            isRestart = true;
        }

        start:while (isFirst || isRestart ||url != null || requestSameListUrl <= 1){
            List<CnkiData> list = new ArrayList<>();
            if(isFirst){
                html.setOrignUrl(getYearInfo);
                http.simpleGet(html);
                url = "http://kns.cnki.net/kns/brief/brief.aspx?keyValue=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD&S=1&pagename=" + html.getContent();
                isFirst = false;
            }else if(isRestart) {
                logger.info("重新启动程序。");
                html = new HtmlInfo();
                html.setEncode("UTF-8");
                html.setUa(ua);
                html.setOrignUrl(getYearInfo);
                http.simpleGet(html);
                isRestart = false;
            }
            logger.info("列表URL:" + url);
            String listUrl = url;
            html.setOrignUrl(url);
            http.simpleGet(html);
            url = null;

            DocumentFragment listNode = DomTree.getNode(html.getContent(),html.getEncode());
            NodeList nextUrlNodeList = DomTree.commonList(CnkiXpath.NEXT_PAGE_URL,listNode);
            if(nextUrlNodeList.getLength() > 0){
                url = urlPrefix + "/kns/brief/brief.aspx" + nextUrlNodeList.item(0).getTextContent();
                logger.info("下一页列表url为：" + url);
            }
            if(url != null){
                requestSameListUrl = 0;
            }

            if(requestSameListUrl <= 1 && url == null){
                logger.info("获取下一页url失败，重新获取下一页url");
                url = listUrl;
                requestSameListUrl = requestSameListUrl + 1;
                isFirst = false;
                isRestart = true;
            }
            NodeList paperUrlNodeList = DomTree.commonList(CnkiXpath.PAPER_URL,listNode);
            if(paperUrlNodeList != null){

                for(int i = 0;i < paperUrlNodeList.getLength();i++){
                    try{
                        logger.info("15s后开始获取第" + (i + 1) + "篇文章信息");
                        Thread.sleep(15 * 1000);

                        CnkiData cnkiData = new CnkiData();

                        //获取文章的发表时间
                        String paperPubTime = null;
                        NodeList paperPubYearNodeList = DomTree.commonList(CnkiXpath.PAPER_PUB_TIME.replace("<n>",i + 2 + ""),listNode);
                        if(paperPubYearNodeList != null && paperPubYearNodeList.getLength() > 0){
                            paperPubTime = paperPubYearNodeList.item(0).getTextContent().replace("\n      ","");
                            cnkiData.setPaper_pub_time(paperPubTime);
                        }

                        String paperUrl = urlPrefix + paperUrlNodeList.item(i).getTextContent();
                        html.setReferUrl(html.getOrignUrl());
                        html.setOrignUrl(paperUrl);
                        http.simpleGet(html);

                        DocumentFragment paperNode = DomTree.getNode(html.getContent(),html.getEncode());
                        //获取标题
                        NodeList titleNodeList = DomTree.commonList(CnkiXpath.TITLE,paperNode);
                        String title = "";
                        if(titleNodeList != null && titleNodeList.getLength() > 0){
                            title = titleNodeList.item(0).getTextContent();
                        }
                        //获取摘要
                        NodeList briefNodeList = DomTree.commonList(CnkiXpath.BRIEF,paperNode);
                        String brief = "";
                        if(briefNodeList != null && briefNodeList.getLength() > 0){
                            brief = briefNodeList.item(0).getTextContent();
                        }
                        //获取作者名字
                        NodeList authorNodeList = DomTree.commonList(CnkiXpath.AUTHOR,paperNode);
                        String author = "";
                        if(authorNodeList != null){
                            for(int j = 0;j < authorNodeList.getLength();j++){
                                author = author + authorNodeList.item(j).getTextContent().replace("\n                    ","") + ";";
                            }
                        }
                        cnkiData.setMd5(MD5Util.MD5(title + author));
                        if(allMD5 != null && allMD5.contains(cnkiData.getMd5())){
                            logger.info("该文章已在数据库中，跳过采集");
                            continue;
                        }
                        //获取文章关键词
                        NodeList keywordsNodeList = DomTree.commonList(CnkiXpath.KEYWORDS,paperNode);
                        String keywords = "";
                        if(keywordsNodeList != null){
                            for(int j = 0;j < keywordsNodeList.getLength();j++){
                                String keyword = keywordsNodeList.item(j).getTextContent();
                                if(keyword.contains("关键词")){
                                    keywords = keywordsNodeList.item(j).getTextContent().replace("\n                  ","");
                                    break;
                                }
                            }
                        }
                        //获取文章下载量
                        NodeList paperDownloadNodeList = DomTree.commonList(CnkiXpath.PAPER_DOWNLOAD,paperNode);
                        String paperDownload = "";
                        if(paperDownloadNodeList != null && paperDownloadNodeList.getLength() > 0){
                            paperDownload = paperDownloadNodeList.item(0).getTextContent();
                        }
                        //获取期刊名称
                        String journalName = "";
                        NodeList journalNodeList = DomTree.commonList(CnkiXpath.JOURNAL_NAME,paperNode);
                        if(journalNodeList != null && journalNodeList.getLength() > 0){
                            journalName = journalNodeList.item(0).getTextContent();
                        }
                        //获取引用本文作者相关的信息+
                        NodeList aboutCiteAuthorNodeList = DomTree.commonList(CnkiXpath.GET_TABLENAME_FILENAME,paperNode);
                        String citeAuthor = "";
                        String tableNameFileName = null;
                        if(aboutCiteAuthorNodeList != null && aboutCiteAuthorNodeList.getLength() > 0){
                            logger.info("15s后开始获取引用本文的作者的信息");
                            Thread.sleep(15 * 1000);
                            tableNameFileName = aboutCiteAuthorNodeList.item(0).getTextContent().split("\\?")[1];
                            html.setOrignUrl(citeAuthorUrl + tableNameFileName);
                            http.simpleGet(html);
                            DocumentFragment citeAuthorNode = DomTree.getNode(html.getContent(),html.getEncode());
                            NodeList citeAuthorNodeList = DomTree.commonList(CnkiXpath.CITE_AUTHOR,citeAuthorNode);
                            if(citeAuthorNodeList != null){
                                for (int j = 0;j < citeAuthorNodeList.getLength();j++){
                                    citeAuthor = citeAuthor + citeAuthorNodeList.item(j).getTextContent() + ";";
                                }
                            }
                        }
                        //获取文章年度被引数
                        String yearsCiteNum = "";
                        if(tableNameFileName != null){
                            logger.info("15s后获取本文的年度被引数");
                            Thread.sleep(15 * 1000);
                            html.setOrignUrl(yearsCiteNumUrl + tableNameFileName);
                            http.simpleGet(html);

                            yearsCiteNum = DomAnalysisXml(html.getContent().replace("text=\"中国学术期刊网络出版总库\"",""));
                        }

                        //获取第一作者的信息
                        NodeList firstAuthorNodeList = DomTree.commonList(CnkiXpath.FIRST_AUTHOR,paperNode);
                        String orgn = "";//作者机构
                        String pubNumAndDownloadNum = "";//作者发文数和下载数
                        if(firstAuthorNodeList.getLength() > 0){
                            String[]firstAuthorNameAndCode = firstAuthorNodeList.item(0).getTextContent().split("\\(")[1].split("\\)")[0].split(",");
                            logger.info("15s后开始获取本文第一作者信息");
                            Thread.sleep(15 * 1000);
                            html.setOrignUrl(
                                    firstAuthorUrl.replace("<authorName>",firstAuthorNameAndCode[1].replace("'","").replace("  "," ").replace(" ","+"))
                                            .replace("<authorCode>",firstAuthorNameAndCode[2].replace("'",""))
                            );
                            http.simpleGet(html);
                            DocumentFragment firstAuthorNode = DomTree.getNode(html.getContent(),html.getEncode());
                            //获取作者机构
                            NodeList orgnNodeList = DomTree.commonList(CnkiXpath.ORGN,firstAuthorNode);
                            if(orgnNodeList != null && orgnNodeList.getLength() > 0){
                                orgn = orgnNodeList.item(0).getTextContent();
                            }
                            //获取作者下载数与发文数
                            NodeList downloadAndPubNumNodeList = DomTree.commonList(CnkiXpath.PUBNUM_DOWNLOADNUM,firstAuthorNode);
                            if(downloadAndPubNumNodeList != null){
                                for(int j = 0;j < downloadAndPubNumNodeList.getLength();j++){
                                    pubNumAndDownloadNum = pubNumAndDownloadNum + downloadAndPubNumNodeList.item(j).getTextContent()
                                            .replace("\n                    ","") + ";";
                                }
                            }
                        }
                        //获取journal的url信息
                        NodeList journalUrlNodeList = DomTree.commonList(CnkiXpath.JOURNAL_URL,paperNode);
                        String journalAffectoi = "";//期刊影响因子
                        String journalDownloadNumAndCiteNum = "";//期刊的总下载量和总引用量
                        if(journalUrlNodeList != null && journalUrlNodeList.getLength() > 0){
                            String [] journalInfo = journalUrlNodeList.item(0).getTextContent()
                                    .split("\\(")[1].split("\\)")[0].split(",");
                            logger.info("15s后开始获取本文的期刊信息");
                            Thread.sleep(15 * 1000);
                            html.setOrignUrl(journalUrl.replace("<1>",journalInfo[1].replace("'",""))
                                    .replace("<2>",journalInfo[3].replace("'",""))
                            );
                            http.simpleGet(html);
                            DocumentFragment journalNode = DomTree.getNode(html.getContent(),html.getEncode());
                            //获取期刊影响因子
                            NodeList journalAffectoiNodeList = DomTree.commonList(CnkiXpath.JOURNAL_AFFECTOI,journalNode);
                            if(journalAffectoiNodeList != null){
                                for(int j = 0;j < journalAffectoiNodeList.getLength();j++){
                                    journalAffectoi = journalAffectoi + journalAffectoiNodeList.item(j).getTextContent() + ";";
                                    if(j == 1){
                                        break;
                                    }
                                }
                            }
                            //获取期刊总下载数和总被引用数
                            NodeList jounalDownloadNumAndCiteNumNodeList = DomTree.commonList(CnkiXpath.JOURNAL_DOWNLOADNUM_CITENUM,journalNode);
                            if(jounalDownloadNumAndCiteNumNodeList != null){
                                for(int j = 0;j < jounalDownloadNumAndCiteNumNodeList.getLength();j++){
                                    journalDownloadNumAndCiteNum = journalDownloadNumAndCiteNum + jounalDownloadNumAndCiteNumNodeList.item(j).getTextContent() + ";";
                                }
                            }
                        }
                        cnkiData.setTitle(title.replace("null",""));
                        cnkiData.setAuthor(author.replace("null",""));
                        cnkiData.setAuthor_affiliation(orgn.replace("null",""));
                        cnkiData.setBrief(brief.replace("null",""));
                        cnkiData.setKeywords(keywords.replace("null",""));
                        cnkiData.setPaper_download_num(paperDownload != null&&!"".equals(paperDownload)?Integer.parseInt(paperDownload):-1);
                        cnkiData.setYears_cite_num(yearsCiteNum!=null ? yearsCiteNum.replace("null","") : "");
                        cnkiData.setCiteAuthors(citeAuthor.replace("null",""));
                        cnkiData.setAuthor_pub_download_num(pubNumAndDownloadNum.replace("null",""));
                        cnkiData.setJournal_affectoi(journalAffectoi.replace("null",""));
                        cnkiData.setJournal_download_cite_num(journalDownloadNumAndCiteNum.replace("null",""));
                        cnkiData.setJournal(journalName.replace("null",""));
                        list.add(cnkiData);
                    }catch (NullPointerException e){
                        logger.info("空指针异常，重新采集该文章信息");
                        i--;
                        //url = listUrl;
                        //isRestart = true;
                       // Thread.sleep(15 *1000);
                        continue ;
                    }
                }
                logger.info("列表文章信息获取完毕，存入数据库");
                CnkiSaveDataToSql.saveCnkiPaperToSql(list);
                for(int i = 0;i<list.size();i++){
                    allMD5+=list.get(i).getMd5();
                }
                logger.info("数据存储完毕，15s后开始获取下一页数据。");
                Thread.sleep(15 * 1000);
            }
        }
    }
    public static String DomAnalysisXml(String yearsCiteNumXml){
        //年度被引数
        String yearsCiteNum = null;
        //创建一个DocumentBuilderFactory的对象
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        //创建DocumentBuilder对象
        DocumentBuilder db = null;
        try {
            InputStream is = new ByteArrayInputStream(yearsCiteNumXml.getBytes());
            db = dbf.newDocumentBuilder();
            //通过DocumentBuilder对象的parser方法加载XML字符串
            Document document = db.parse(is);
            //获取所有Item节点的集合
            NodeList itemList = document.getElementsByTagName("Item");
            //遍历每一个Item节点
            for (int i = 0; i < itemList.getLength(); i++) {
                //通过 item(i)方法 获取一个Item节点，nodelist的索引值从0开始
                Node item = itemList.item(i);
                //解析Item节点的子节点
                NodeList childNodes = item.getChildNodes();
                String year = null;
                String citing = null;
                String subCiting = null;
                //遍历childNodes获取每个节点的节点名和节点值
                for (int k = 0; k < childNodes.getLength(); k++) {
                    //区分出text类型的node以及element类型的node
                    if (childNodes.item(k).getNodeType() == Node.ELEMENT_NODE) {
                        if("YEAR".equals(childNodes.item(k).getNodeName())){
                            year = childNodes.item(k).getFirstChild().getNodeValue();
                        }else if("CITING".equals(childNodes.item(k).getNodeName())){
                            citing = childNodes.item(k).getFirstChild().getNodeValue();
                        }else if("SUB_CITING".equals(childNodes.item(k).getNodeName())){
                            subCiting = childNodes.item(k).getFirstChild().getNodeValue();
                        }
                    }
                }
                yearsCiteNum = yearsCiteNum + year + ":" + citing + "," + subCiting + ";";
            }
            return yearsCiteNum;
        } catch (ParserConfigurationException e) {
            e.printStackTrace();
        } catch (SAXException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
}


